package com.lzx.schedule;


import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.lzx.entity.Ip;
import com.lzx.entity.Proxywebsite;
import com.lzx.service.IpService;
import com.lzx.util.CrawlerUtil;
import org.apache.http.HttpEntity;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;

import java.util.LinkedList;
import java.util.List;

@Service
public class AsyncService {

    @Autowired
    IpService ipService;


    @Async("crawlerExecutor")
    public void crawlerIp(Proxywebsite site) throws Exception{
        for(int j = site.getSiteindex(); j < site.getSitecount();j++){
            String url = (site.getUrl()).replace("*",""+j);
            System.out.println("要爬的网站：" + url);
            List<Ip> ips = getIP(url);
            System.out.println("获取的ip有"+ips.size()+"个，分别是:" + ips);

            testIP(ips);
            System.out.println("有效的ip有"+ips.size()+"个，分别是:" + ips);

            for(int num =0;num < ips.size();num++){
                Ip ip = ips.get(num);
                QueryWrapper<Ip> queryWrapper = new QueryWrapper<>();
                queryWrapper.eq("url",ip.getUrl());
                if(ipService.getMap(queryWrapper)!=null){
                    ips.remove(num);
                    System.out.println("出现重复：" + ip);
                }

            }
            if(ips != null && ips.size() != 0)
                ipService.saveOrUpdateBatch(ips);

            Thread.sleep(1000*5);

        }

    }
    public List<Ip>  getIP(String url){
        List<Ip> ips = new LinkedList<Ip>();
        CloseableHttpResponse response = null;
        CloseableHttpClient httpClient = null;
        try{

            HttpGet get = new HttpGet(url);
            get.setHeader("user-agent"," Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36");

            RequestConfig config = RequestConfig.custom().setConnectTimeout(500).build();
            get.setConfig(config);

            httpClient = HttpClientBuilder.create().build();
            response = httpClient.execute(get);

            if(response.getStatusLine().getStatusCode() == 200) {
                HttpEntity entity = response.getEntity();
                String html = EntityUtils.toString(entity,"gb2312");
                Document doc = Jsoup.parse(html);
                Elements ulList = doc.select("table");
                if(ulList.size() == 0){
                    return ips;
                }
                int maxIndex = 0;
                int max = ulList.get(0).select("tr").size();
                for(int i = 1;i < ulList.size();i++){
                    int count = ulList.get(i).select("tr").size();
                    if(count > max){
                        max = count;
                        maxIndex = i;
                    }
                }
                Elements liList = ulList.get(maxIndex).select("tr");

                for(int i = 1;i < liList.size();i++) {
                    String [] strings = liList.get(i).text().split(" ");
                    Ip ip = new Ip();
                    ip.setUrl(strings[0]);
                    ip.setPort(new Integer(strings[1]));
                    ips.add(ip);
                }
                EntityUtils.consume(response.getEntity());
            }else{
                EntityUtils.consume(response.getEntity());
            }

        }catch (Exception e){
            e.printStackTrace();
        }finally {
            try{
                if(response != null){
                    response.close();
                }
                if(httpClient != null){
                    httpClient.close();
                }
            }catch (Exception e){
                e.printStackTrace();
            }
        }
        return ips;
    }
    public List<Ip> testIP(List<Ip> ips){
        for(int i = 0;i < ips.size();){
            if(!CrawlerUtil.runTest(ips.get(i),"https://www.csdn.net/")){
                ips.remove(i);
            }else {
                System.out.println("有效" + ips.get(i));
                i++;
            }
        }
        return ips;
    }


}
